#include "config.h"

#define DBG_SUBSYS S_LIBINTERFACE

#include "lich_api.h"

#include "storage.h"
#include "volume.h"
#include "license.h"
#include "option.h"
#include "utils.h"
#include "hostmap.h"
#include "sysy_lib.h"


#define LICHBD "lichbd"
#define MAX_SUBCMD_DEPTH 8
#define CMD_NEED_NULL (1 << 0)
#define CMD_NEED_ARG (1 << 1)
#define CMD_NEED_ROOT (1 << 2)

// insert line break
#define CMD_BR "BR"

int object_md5sum(const fileid_t *oid);
struct command {
        const char *name;
        const struct subcommand *sub;
        int (*parser)(int, const char *);
};

struct subcommand {
        const char *name;
        const char *arg;
        const char *opts;
        const char *desc;
        const struct subcommand *sub;
        unsigned long flags;
        int (*fn)(int, char **);
        const struct sd_option *options;
};

static const char program_name[] = "lichbd";
static char system_cmd[] = "path";
static bool found_system_cmd =   false;

int subcmd_depth = -1;
struct subcommand *subcmd_stack[MAX_SUBCMD_DEPTH];

static const struct sd_option lichbd_options[] = {
        {'v', "verbose", false, "print more information than default", NULL},
        {'h', "help", false, "display this help and exit", NULL},
        { 0, NULL, false, NULL, NULL},
};

/*static struct pool_cmd_data {*/
/*} pool_cmd_data;*/

static struct volume_cmd_data {
        int full; // vol info full
        int force;
        int output_format;
        // 0: raw, 1: row2, 2: lsv
        int volume_format;
        int mult_thread;
        int concurrent;
        int pipeline;
        int pithy;
        uint64_t offset;
        __off_t length;
        size_t size;
        uint32_t repnum;
        char storage_area[MAX_NAME_LEN];
        char pool_name[MAX_NAME_LEN];
        char protocol_name[MAX_NAME_LEN];
        char volume_name[MAX_NAME_LEN];
        char path[MAX_NAME_LEN];
        ec_t ec;
} volume_cmd_data;

static void __volume_cmd_dump(struct volume_cmd_data *cmd)
{
        DBUG("path %s volume_format %d size %ld full %d force %d\n",
             cmd->path,
             cmd->volume_format,
             cmd->size,
             cmd->full,
             cmd->force);
}

struct cmd_flag_data {
        bool all_flag;
        bool mult_thread_flag;
        bool pipeline_flag;
        bool size_flag;
        bool pool_name_flag;
        bool protocol_name_flag;
        bool volume_name_flag;

        bool path_flag;
        bool output_format_flag;
        bool storage_area_flag;
        bool thin_flag;
        bool recursive_flag;

        bool repnum_flag;
        bool offset_flag;
        bool length_flag;
        bool pithy_flag;
        char priority;
        int output_format;
        int arguments;
} cmd_flag_data = {
        false, false, false, false, false, false,
        false, false, false, false, false,
        false, false, false, false, false,
        -1, 0, 0};

static struct pool_cmd_data {
        char protocol_name[MAX_NAME_LEN];
} pool_cmd_data;

static struct snap_cmd_data {
        int all;
        int output_format;
        char storage_area[MAX_NAME_LEN];
        char protocol_name[MAX_NAME_LEN];
        char volume_name[MAX_NAME_LEN];
} snap_cmd_data;

/*
static struct attr_cmd_data {
    char protocol_name[MAX_NAME_LEN];
} attr_cmd_data;
 */


void subcommand_usage(char *cmd, char *subcmd, int status);

static int (*command_parser)(int, const char *);
static int (*command_fn)(int, char **);
static const char *command_opts;
static const char *command_arg;
static const char *command_desc;
static const struct sd_option *command_options;

static int __is_valid_pool(const char *str)
{
        if (strlen(str) == 0)
                return 0;

        if (str[0] == '/' || str[strlen(str) - 1] == '/') {
                return 0;
        }

        if (_valid_pool(str))
                return 0;

        return 1;
}

#if 0
static int __is_valid_volume(const char *str)
{
        if (strlen(str) == 0)
                return 0;

        if (str[0] == '/' || str[strlen(str) - 1] == '/') {
                return 0;
        }

        return 1;
}
#endif

static int __protocol_trans(const char *_protocol, char *path)
{
        int ret;
        char protocol[MAX_NAME_LEN];

        if (_protocol && strlen(_protocol)) {
                strcpy(protocol, _protocol);
        } else {
                strcpy(protocol, gloconf.default_protocol);
        }

        if (strcmp(protocol, "lichbd") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", gloconf.lichbd_root);
        } else if (strcmp(protocol, "nbd") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", gloconf.nbd_root);
        } else if (strcmp(protocol, "sheepdog") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", gloconf.sheepdog_root);
        } else if (strcmp(protocol, "iscsi") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", "iscsi");
        } else if (strcmp(protocol, "nfs") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", "nfs");
#ifdef NVMF
        } else if (strcmp(protocol, "nvmf") == 0) {
                snprintf(path, MAX_NAME_LEN, "%s", "nvmf");
#endif
        } else {
                ret = EINVAL;
                fprintf(stderr, "lichbd: protocol[%s] was invalid.\n", protocol);
#ifdef NVMF
                fprintf(stderr, "supported protocol: iscsi, lichbd, nbd, nfs, nvmf.\n");
#else
                fprintf(stderr, "supported protocol: iscsi, lichbd, nbd, nfs.\n");
#endif
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __create_protocol(const char *pool, const char *_protocol)
{
        int ret, retry = 0;
        char tmp[MAX_NAME_LEN];
        char protocol[MAX_PATH_LEN];

        ret = __protocol_trans(_protocol, protocol);
        if (ret) {
                GOTO(err_ret, ret);
        }

        snprintf(tmp, MAX_NAME_LEN, "/%s", protocol);

retry:
        ret = utils_mkdir(pool, tmp, 0, NULL, "");
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                } else if (ret == EAGAIN) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else if (ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int path_prep_for_pool(const char *path, char *pool, char *protocol)
{
        int ret;
        char _protocol[MAX_NAME_LEN];

        ret = path_decode(path, pool, _protocol, NULL, NULL);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

/*
        ret = __protocol_trans(_protocol, protocol);
        if (ret) {
                GOTO(err_ret, ret);
        }
*/

        if (!strlen(pool)) {
                ret = EINVAL;
                GOTO(err_ret, ret);
        }

        snprintf(protocol, MAX_NAME_LEN, "/%s", _protocol);

        return 0;
err_ret:
        return ret;
}

static int path_prep_for_volume(const char *path, char *pool, char *protocol, char *volume)
{
        int ret;
        char vol[MAX_NAME_LEN], snap[MAX_NAME_LEN] = "",
             _volume[MAX_PATH_LEN];

        ret = path_decode(path, pool, protocol, vol, snap);
        if (ret) {
                GOTO(err_ret, ret);
        }

/*
        ret = __protocol_trans(buf, protocol);
        if (ret) {
                GOTO(err_ret, ret);
        }
*/

        if (!strlen(pool)) {
                ret = EINVAL;
                GOTO(err_ret, ret);
        }

        if (strlen(snap))
                snprintf(_volume, MAX_NAME_LEN, "/%s/%s@%s", protocol, vol, snap);
        else
                snprintf(_volume, MAX_NAME_LEN, "/%s/%s", protocol, vol);

        path_normalize(_volume, volume);

        DINFO("create %s\n", volume);

        return 0;
err_ret:
        return ret;
}

static int path_prep_for_head(const char *path, char *pool, char *path2) {
        int ret;
        char _path2[MAX_PATH_LEN];

        ret = path_head(path, '/', pool, _path2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        if (!strlen(pool)) {
                ret = EINVAL;
                GOTO(err_ret, ret);
        }

        path_normalize(_path2, path2);

        return 0;
err_ret:
        return ret;
}

/**
 *
 * @param argc the number of all cmd line, include cmd
 * @param nparam 调节单选项，和带参数的选项
 * @return
 */
static int __check_arguments(int argc, int nparam)
{
        int ret;
        int arguments = 0;

        arguments = 3 + cmd_flag_data.arguments * 2 + nparam;

        if (argc < arguments) {
                ret = EINVAL;
                fprintf(stderr, "lichbd: too few arguments(%d,%d)\n", argc, arguments);
                GOTO(err_ret, ret);
        } else if (argc > arguments) {
                ret = EINVAL;
                fprintf(stderr, "lichbd: too more arguments\n");
                GOTO(err_ret, ret);
        }
        return 0;
err_ret:
        return ret;
}

static int __stor_mkdir(int argc, char **argv)
{
        int ret;
        const char *path;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        path = argv[argc - 1];

        char pool[MAX_PATH_LEN];
        char path2[MAX_PATH_LEN];
        ret = path_prep_for_head(path, pool, path2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        char protocol[MAX_PATH_LEN];
        char dir[MAX_PATH_LEN];
        ret= path_prep_for_head(path2, protocol, dir);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(pool, protocol);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (!strcmp(protocol, "iscsi")) {
                fprintf(stderr, "\x1b[1;31miscsi root should not create dir\x1b[0m\n");
                ret = EPERM;
                GOTO(err_ret, ret);
        }

        ret = utils_mkdir(pool, path2, 1, NULL, "");
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

static int __stor_rmdir(int argc, char **argv)
{
        int ret;
        const char *path;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        path = argv[argc - 1];

        char pool[MAX_PATH_LEN];
        char dir[MAX_PATH_LEN];
        ret = path_prep_for_head(path, pool, dir);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = utils_rmdir(pool, dir);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

static int __stor_lsdir(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN], dir[MAX_PATH_LEN];

        if (argc % 2 )
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_pool(_path, pool, protocol);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(pool, protocol + 1);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        ret = path_prep_for_head(_path, pool, dir);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_list(pool, dir, 2, DEFAULT_FORMAT);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else if (ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_mkpool(int argc, char **argv)
{
        int ret, retry = 0;
        const char *pool;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        pool = argv[argc - 1];

        if (!__is_valid_pool(pool)) {
                ret = EINVAL;
                fprintf(stderr, "pool was invalid\n");
                GOTO(err_ret, ret);
        }

retry:
        ret = utils_pool_create(pool);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_attrset(int argc, char **argv)
{
        int ret, retry = 0;
        const char *_key, *_value;
        char *_path = NULL;

        ret = __check_arguments(argc, 3);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path=argv[argc - 3];
        _key=argv[argc - 2];
        _value=argv[argc - 1];

        ret = storage_license_check();
        if (unlikely(ret)) {
                fprintf(stderr,"error: %s\n", strerror(ret));
                GOTO(err_ret, ret);
        }

        char pool[MAX_PATH_LEN];
        char path2[MAX_PATH_LEN];
        ret = path_prep_for_head(_path, pool, path2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = lich_xattr_set(pool, path2, _key, _value);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_attrlist(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        char pool[MAX_PATH_LEN];
        char path2[MAX_PATH_LEN];
        ret = path_prep_for_head(_path, pool, path2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = lich_xattr_list(pool, path2);
        if (unlikely(ret)) {
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_attrget(int argc, char **argv)
{
        int ret, retry = 0;
        const char *_key;
        char *_path = NULL;

        ret = __check_arguments(argc, 2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 2];
        _key = argv[argc - 1];

        char pool[MAX_PATH_LEN];
        char path2[MAX_PATH_LEN];
        ret = path_prep_for_head(_path, pool, path2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = lich_xattr_get(pool, path2, _key, cmd_flag_data.output_format);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_attrremove(int argc, char **argv)
{
        int ret, retry = 0;
        const char *_key;
        char *_path = NULL;

        ret = __check_arguments(argc, 2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 2];
        _key = argv[argc - 1];

        char pool[MAX_PATH_LEN];
        char path2[MAX_PATH_LEN];
        ret = path_prep_for_head(_path, pool, path2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = lich_xattr_remove(pool, path2, _key);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

/**
 * @pre pool内有卷时不允许删除
 *
 * @post etcd上记录被清除
 * @post 各个节点池内盘上信息被清除
 *
 * @see
 *
 * @param argc
 * @param argv
 * @return
 */
static int __stor_rmpool(int argc, char **argv)
{
        int ret, retry = 0;
        const char *pool = NULL;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        pool = argv[argc - 1];

        if (!__is_valid_pool(pool)) {
                ret = EINVAL;
                fprintf(stderr, "pool was invalid\n");
                GOTO(err_ret, ret);
        }

retry:
        ret = utils_pool_remove(pool);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == ENOENT || ret == ENOKEY) {
                        ret = ENOENT;
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                                DBUG("remove %s\n", pool);
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_lspools(int argc, char **argv)
{
        (void) argc;
        (void) argv;

        return utils_pool_list(0);
}

static int __stor_create(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], volume[MAX_NAME_LEN];

        vol_param_t param;

        if (cmd_flag_data.size_flag == false) {
                ret = EINVAL;
                fprintf(stderr, "lichbd: too few arguments\n");
                GOTO(err_ret, ret);
        } else {
                if (cmd_flag_data.volume_name_flag)
                        ret = __check_arguments(argc, 0);
                else
                        ret = __check_arguments(argc, 1);
        }

        if (ret) {
                GOTO(err_ret, ret);
        }

        ret = storage_license_check();
        if (unlikely(ret)) {
                fprintf(stderr,"error: %s\n", strerror(ret));
                GOTO(err_ret, ret);
        }

        if (cmd_flag_data.priority != -1 &&
                        cmd_flag_data.priority != 0 &&
                        cmd_flag_data.priority != 1) {
                ret = EINVAL;
                fprintf(stderr, "lichbd: too more arguments\n");
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        if (cmd_flag_data.repnum_flag &&
                        (volume_cmd_data.repnum > LICH_REPLICA_MAX
                         || volume_cmd_data.repnum < LICH_REPLICA_MIN)) {
                fprintf(stderr, "lichbd: repnum was invalid\n");
                GOTO(err_ret, ret);
        }

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(pool, protocol);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        vol_param_init(&param);

        param.volume_format = volume_cmd_data.volume_format;
        param.priority = cmd_flag_data.priority;
        param.repnum = volume_cmd_data.repnum;
        param.size = volume_cmd_data.size;
        strcpy(param.site, volume_cmd_data.storage_area);

retry:
        ret = utils_touch_with_area(pool, volume, &param);
        if (ret) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        /*
        ret = md_chunk_setsite_path(pool, volume, volume_cmd_data.storage_area);
        if (ret)
                GOTO(err_ret, ret);
        */

        printf("create %s ok, size: %ju\n", volume, volume_cmd_data.size);

        return 0;
err_ret:
        return ret;
}

/**
 * 删除卷的过程分两个阶段：
 * - 先移入/system/unlink
 * - 后台线程异步回收
 *
 * @param argc
 * @param argv
 * @return
 */
static int __stor_rm(int argc, char **argv)
{
        int ret, retry = 0, arg = 0;
        char *_path = NULL, volume[MAX_NAME_LEN], pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN];

        if (volume_cmd_data.force)
                arg++;

        if (cmd_flag_data.volume_name_flag)
                ret = __check_arguments(argc, 0 + arg);
        else
                ret = __check_arguments(argc, 1 + arg);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        __volume_cmd_dump(&volume_cmd_data);

retry:
        ret = utils_rmvol(pool, volume, volume_cmd_data.force);
        if (ret) {
                if (ret == EPERM) {
                        GOTO(err_ret, ret);
                }

                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_ls(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN];

        if (argc % 2 )
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_pool(_path, pool, protocol);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(pool, protocol + 1);
        if (unlikely(ret)) {
                if (ret == ENOSPC)
                        return 0;
                else
                        GOTO(err_ret, ret);
        }

retry:
        ret = utils_list(pool, protocol, 0, volume_cmd_data.output_format);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_info(int argc, char **argv)
{
        int ret, arg = 1, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN];

        if (cmd_flag_data.volume_name_flag)
                arg = 0;
        //if (volume_cmd_data.full)
        //        arg++;

        ret  = __check_arguments(argc, arg);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);
retry:
        ret = utils_info(pool, volume, volume_cmd_data.full, volume_cmd_data.output_format);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_allocate(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN];

        if (cmd_flag_data.volume_name_flag)
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);
retry:
        ret = utils_allocate(pool, volume, volume_cmd_data.pipeline,
                        volume_cmd_data.concurrent, 0);
        if (unlikely(ret)) {
                if (ret == ECANCELED)
                        GOTO(err_ret, ret);
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_resize(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN];

        if (cmd_flag_data.volume_name_flag)
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (volume_cmd_data.size <= 0 || volume_cmd_data.size % LICH_BLOCK_SPLIT != 0) {
                fprintf(stderr, "size must be greater than zero and aligned by %d Bytes!\n", LICH_BLOCK_SPLIT);
                ret = EINVAL;
                GOTO(err_ret, ret);
        }
retry:
        ret = utils_truncate(pool, volume, volume_cmd_data.size);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        printf("resize %s new size %ju ok\n", volume, volume_cmd_data.size);

        return 0;
err_ret:
        return ret;
}

/**
 * @brief 全量copy, 通过lichbd_read/write实现
 *
 * @note 如目标卷存在？ 大小匹配/大小不匹配
 * @note 如目标卷不存在？
 *
 * @note 不复制hostmap
 *
 * @param argc
 * @param argv
 * @return
 */
static int __stor_copy(int argc, char **argv)
{
        int ret, retry = 0;
        char *_from = NULL, *_to = NULL,
             pool1[MAX_NAME_LEN], pool2[MAX_NAME_LEN],
             protocol1[MAX_NAME_LEN], protocol2[MAX_NAME_LEN],
             from[MAX_NAME_LEN], to[MAX_NAME_LEN];

        if (cmd_flag_data.mult_thread_flag)
                ret = __check_arguments(argc, 3);
        else
                ret = __check_arguments(argc, 2);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _from = argv[argc - 2];
        _to = argv[argc - 1];

        ret = path_prep_for_volume(_from, pool1, protocol1, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = path_prep_for_volume(_to, pool2, protocol2, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(pool2, protocol2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = stor_copy_with_area(pool1, from, pool2, to, volume_cmd_data.mult_thread, cmd_flag_data.thin_flag,
                        cmd_flag_data.priority, volume_cmd_data.storage_area, volume_cmd_data.volume_format);

        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        /**if (to[0] != ':') {
                ret = md_chunk_setsite_path(pool2, to, volume_cmd_data.storage_area);
                if (ret)
                        GOTO(err_ret, ret);
        }**/

        return 0;
err_ret:
        return ret;
}

static int __stor_export(int argc, char **argv)
{
        int ret, retry = 0;
        char from[MAX_NAME_LEN], to[MAX_NAME_LEN];
        char pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN];
        char *_from = NULL, *_to = NULL;

        ret = __check_arguments(argc, 2);
        if (ret)
                GOTO(err_ret, ret);

        _from = argv[argc - 2];
        _to = argv[argc - 1];

        ret = path_prep_for_volume(_from, pool, protocol, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (_to[0] == '/' || _to[0] == '.') {
                strcpy(to, ":");
                strcat(to, _to);
        retry1:
                ret = stor_copy(pool, from, to, volume_cmd_data.mult_thread, 0, cmd_flag_data.priority);
                if (unlikely(ret)) {
                        if (ret == EAGAIN || ret == ENOSPC) {
                                USLEEP_RETRY(err_ret, ret, retry1, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                        } else
                                GOTO(err_ret, ret);
                }
        } else if (_to[0] == '-') {
        retry2:
                ret = utils_cat(pool, from, 0, 0);
                if (unlikely(ret)) {
                        if (ret == EAGAIN || ret == ENOSPC) {
                                USLEEP_RETRY(err_ret, ret, retry2, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                        } else
                                GOTO(err_ret, ret);
                }
        } else {
                ret = EINVAL;
                fprintf(stderr, "path is invalid\n");
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_import(int argc, char **argv)
{
        int ret, retry = 0;
        char from[MAX_NAME_LEN], to[MAX_NAME_LEN];
        char pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN];
        char *_from = NULL, *_to = NULL;

        ret = __check_arguments(argc, 2);
        if (ret)
                GOTO(err_ret, ret);

        _to = argv[argc - 1];
        _from = argv[argc - 2];

        ret = path_prep_for_volume(_to, pool, protocol, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry0:
        ret = __create_protocol(pool, protocol);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry0, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        if (_from[0] == '/' || _from[0] == '.' || _from[0] == '-') {
                strcpy(from, ":");
                strcat(from, _from);
        } else {
                ret = EINVAL;
                fprintf(stderr, "path is invalid\n");
                GOTO(err_ret, ret);
        }

retry:
        ret = stor_copy_with_area("", from, pool, to, volume_cmd_data.mult_thread, cmd_flag_data.thin_flag,
                                  cmd_flag_data.priority, volume_cmd_data.storage_area, volume_cmd_data.volume_format);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

#if 0
retry1:
        ret = md_chunk_setsite_path(pool, to, volume_cmd_data.storage_area);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry1, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }
#endif

        return 0;
err_ret:
        return ret;
}

static int __stor_mv(int argc, char **argv)
{
        int ret, retry = 0;
        char *_from = NULL, *_to = NULL,
             pool1[MAX_NAME_LEN], pool2[MAX_NAME_LEN],
             protocol1[MAX_NAME_LEN], protocol2[MAX_NAME_LEN],
             from[MAX_NAME_LEN], to[MAX_NAME_LEN];

        ret = __check_arguments(argc, 2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _from = argv[argc - 2];
        _to = argv[argc - 1];

        ret = path_prep_for_volume(_from, pool1, protocol1, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = path_prep_for_volume(_to, pool2, protocol2, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        if (strcmp(pool1, pool2)) {
                fprintf(stderr, "pool must same!\n");
                ret = EINVAL;
                goto err_ret;
        }

        if (strcmp(protocol1, protocol2)) {
                fprintf(stderr, "protocol must same!\n");
                ret = EINVAL;
                goto err_ret;
        }

retry:
        ret = utils_rename(pool1, from, to);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_migrate(int argc, char **argv)
{
        int ret, retry = 0;
        char *_from = NULL, *_to = NULL,
             src_pool[MAX_NAME_LEN], dest_pool[MAX_NAME_LEN],
             protocol1[MAX_NAME_LEN], protocol2[MAX_NAME_LEN],
             from[MAX_NAME_LEN], to[MAX_NAME_LEN], key[MAX_NAME_LEN];
        int arg = 0;

        if (volume_cmd_data.force)
                arg++;

        ret = __check_arguments(argc, 2 + arg);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _from = argv[argc - 2];
        _to = argv[argc - 1];

        ret = path_prep_for_volume(_from, src_pool, protocol1, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = path_prep_for_volume(_to, dest_pool, protocol2, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(dest_pool, protocol2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        etcd_lock_t lock;
        strcpy(key, _from);
        str_replace_char(key, '/', '_');
        ret = utils_migrate_lock(&lock, key);
        if (unlikely(ret))
                GOTO(err_ret, ret);
retry:
        ret = utils_migrate(src_pool, from, dest_pool, to, cmd_flag_data.priority,
                        volume_cmd_data.storage_area, volume_cmd_data.force);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_unlock, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_unlock, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_unlock, ret);
        }

        utils_migrate_unlock(&lock);
        return 0;
err_unlock:
        utils_migrate_unlock(&lock);
err_ret:
        return ret;
}

static int __stor_duplicate(int argc, char **argv)
{
        int ret, retry = 0;
        char *_from = NULL, *_to = NULL,
             src_pool[MAX_NAME_LEN], dest_pool[MAX_NAME_LEN],
             protocol1[MAX_NAME_LEN], protocol2[MAX_NAME_LEN],
             from[MAX_NAME_LEN], to[MAX_NAME_LEN];

        ret = __check_arguments(argc, 2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _from = argv[argc - 2];
        _to = argv[argc - 1];

        ret = path_prep_for_volume(_from, src_pool, protocol1, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = path_prep_for_volume(_to, dest_pool, protocol2, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(dest_pool, protocol2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_duplicate(src_pool, from, dest_pool, to, cmd_flag_data.priority,
                              volume_cmd_data.storage_area, 0);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

// volume locker

static int __stor_get_volume_path(int argc, char **argv, char *pool, char *buf)
{
        int ret;
        char *_path = NULL, protocol[MAX_NAME_LEN];

        if (cmd_flag_data.volume_name_flag)
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, buf);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        return 0;
err_ret:
        return ret;
}

static int __stor_lock(int argc, char **argv)
{
        int ret, buflen, retry = 0;
        char volume[MAX_NAME_LEN], pool[MAX_NAME_LEN];
        char buf[MAX_BUF_LEN];

        ret = __stor_get_volume_path(argc, argv, pool, volume);
        UNLIKELY_GOTO(err_ret, ret);

        ret = lich_xattr_get_buf(pool, volume, LICH_SYSTEM_ATTR_LOCK, buf, &buflen);
        if (ret == 0 && buflen > 0) {
                // already exists
                ret = EPERM;
                GOTO(err_ret, ret);
        }

retry:
        ret = lich_xattr_set(pool, volume, LICH_SYSTEM_ATTR_LOCK, "1");
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_unlock(int argc, char **argv)
{
        int ret, retry = 0;
        char volume[MAX_NAME_LEN], pool[MAX_NAME_LEN];

        ret = __stor_get_volume_path(argc, argv, pool, volume);
        UNLIKELY_GOTO(err_ret, ret);

retry:
        ret = lich_xattr_remove(pool, volume, LICH_SYSTEM_ATTR_LOCK);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_md5sum(int argc, char **argv)
{
        int ret, retry = 0;
        char path[MAX_NAME_LEN], pool[MAX_NAME_LEN];
        char protocol[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        ret = path_prep_for_volume(argv[argc - 1], pool, protocol, path);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_md5sum(pool, path, volume_cmd_data.mult_thread);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_connection(int argc, char **argv)
{
        int ret, retry = 0;
        char path[MAX_NAME_LEN], pool[MAX_NAME_LEN];
        char protocol[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        ret = path_prep_for_volume(argv[argc - 1], pool, protocol, path);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_connection(pool, path);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_createmap(int argc, char **argv)
{
        int ret, retry = 0;
        char *initiator = NULL;
        char path[MAX_NAME_LEN], pool[MAX_NAME_LEN];
        char protocol[MAX_NAME_LEN];

        ret = __check_arguments(argc, 2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        initiator = argv[argc - 2];

        ret = path_prep_for_volume(argv[argc - 1], pool, protocol, path);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = hostmap_create(initiator, pool, path);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_rmmap(int argc, char **argv)
{
        int ret, retry = 0;
        char *initiator = NULL;
        char path[MAX_NAME_LEN], pool[MAX_NAME_LEN];
        char protocol[MAX_NAME_LEN];

        ret = __check_arguments(argc, 2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        initiator = argv[argc - 2];

        ret = path_prep_for_volume(argv[argc - 1], pool, protocol, path);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = hostmap_rm(initiator, pool, path);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_rmall(int argc, char **argv)
{
        int ret, retry = 0;
        char *initiator = NULL;

        ret = __check_arguments(argc, 1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        initiator = argv[argc - 1];

retry:
        ret = hostmap_rm_all(initiator);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_listmap(int argc, char **argv)
{
        int ret, retry = 0;
        char *initiator = NULL;

        ret = __check_arguments(argc, 1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        initiator = argv[argc - 1];

retry:
        ret = hostmap_list(initiator, cmd_flag_data.output_format_flag);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}
#if 0
static int __is_valid_snappath(const char *snappath)
{
        int count = 0;
        char tmp[MAX_NAME_LEN];
        char *list[2];

        count = 2;
        strcpy(tmp, snappath);
        _str_split(tmp, '@', list, &count);
        if (count != 2) {
                return 0;
        }

        return __is_valid_volume(list[0]);
}
#endif

static int __stor_snap_create(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_create(pool, snappath, FALSE, FALSE);
        if (ret) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_ls(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN];

        if (snap_cmd_data.all)
                ret = __check_arguments(argc, 0);
        else
                ret = __check_arguments(argc, 1);

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_list(pool, volume, snap_cmd_data.all, snap_cmd_data.output_format, 0);
        if (ret) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_rm(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_remove(pool, snappath, 0);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_rollback(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_rollback(pool, volume);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EBUSY) {
                        if (retry) {
                                //pass
                        } else
                                GOTO(err_ret, ret);
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, _get_rpc_timeout() * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_protect(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_protect(pool, snappath);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_unprotect(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_unprotect(pool, snappath);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_clone(int argc, char **argv)
{
        int ret, retry = 0;
        char *_from = NULL, *_to = NULL,
             src_pool[MAX_NAME_LEN], dest_pool[MAX_NAME_LEN],
             protocol1[MAX_NAME_LEN], protocol2[MAX_NAME_LEN],
             from[MAX_NAME_LEN], to[MAX_NAME_LEN];

        ret = __check_arguments(argc, 2);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _from = argv[argc - 2];
        _to = argv[argc - 1];


        ret = path_prep_for_volume(_from, src_pool, protocol1, from);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = path_prep_for_volume(_to, dest_pool, protocol2, to);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        ret = __create_protocol(dest_pool, protocol2);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_clone(src_pool, from, dest_pool, to, cmd_flag_data.priority);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else {
                        if (ret == EEXIST && retry) {
                                //pass
                        } else
                                GOTO(err_ret, ret);
                }
        }

#if 0
retry1:
        ret = md_chunk_setsite_path(dest_pool, to, volume_cmd_data.storage_area);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry1, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }
#endif

        return 0;
err_ret:
        return ret;
}


static int __stor_snap_flat(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_flat(pool, snappath);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_cat(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_cat(pool, snappath);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_export(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, *_to = NULL, *_idx = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        if( argc % 2 )
                ret = __check_arguments(argc, 2);
        else
                ret = __check_arguments(argc, 3);

        if (ret) {
                GOTO(err_ret, ret);
        }

        if( argc % 2 ){
                _path = argv[argc - 2];
                _to = argv[argc - 1];
        }else{
                _path = argv[argc - 3];
                _idx = argv[argc - 2];
                _to = argv[argc -1];
        }

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_copy(pool, snappath, _to, _idx);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EEXIST) {
                        if (retry == 0) {
                                GOTO(err_ret, ret);
                        } else {
                        }
                } else if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_snap_diff(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, *_to = NULL, *_idx = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], snappath[MAX_NAME_LEN];

        if( argc % 2 )
                ret = __check_arguments(argc, 2);
        else
                ret = __check_arguments(argc, 3);

        if (ret) {
                GOTO(err_ret, ret);
        }

        if( argc % 2 ){
                _path = argv[argc - 2];
                _to = argv[argc - 1];
        }else {
                _path = argv[argc - 3];
                _idx = argv[argc - 2];
                _to = argv[argc -1];
        }

        ret = path_prep_for_volume(_path, pool, protocol, snappath);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_snapshot_diff(pool, snappath, _to, _idx, 0);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

typedef struct {
        char *key;
        int recursive;
} arg_t;

STATIC int __utils_find(void *pool, void *_arg)
{
        int ret;
        arg_t *arg = _arg;

        ret = utils_find(pool, "/", arg->key, arg->recursive);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_find(int argc, char **argv)
{
        int ret, recursive, has_key = 0, nparam;
        char *_path = NULL, *_key = NULL, path[MAX_PATH_LEN], pool[MAX_NAME_LEN];

        recursive = cmd_flag_data.recursive_flag;
        if (recursive) {
#if 0
                if (argc % 2 == 0) { // lichbd find -r <path>
                        nparam = 1;
                } else { // lichbd find -r <path> <key>
                        nparam = 2;
                        has_key = 1;
                }
#else
                nparam = 1;
#endif
        } else {
#if 0
                if (argc % 2 == 0) { // lichbd find <path> <key>
                        nparam = 1;
                        has_key = 1;
                } else { // lichbd find <path>
                        nparam = 0;
                }
#else
                nparam = 0;
#endif
        }
        ret = __check_arguments(argc, nparam);
        if (ret) {
                GOTO(err_ret, ret);
        }

        if (has_key) {
                _path = argv[argc - 2];
                _key = argv[argc - 1];
        } else {
                _path = argv[argc - 1];
        }

        path_head(_path, '/', pool, path);

        arg_t arg = {
                .key = _key,
                .recursive = recursive ? 0 :1,
        };

        if (!strlen(pool)) {
                system_pool_iterator(__utils_find, &arg);
        } else {
                ret = utils_find(pool, (strlen(path) ? path : "/"), _key, recursive ? 0 : 1);
                if (unlikely(ret)) {
                        GOTO(err_ret, ret);
                }
        }

        return 0;
err_ret:
        fprintf(stderr, "error: %s\n", strerror(ret));
        return ret;
}

static int __stor_stat(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], volume[MAX_NAME_LEN];

        ret = __check_arguments(argc, 0);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_stat(pool, volume, cmd_flag_data.output_format);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_chunkinfo(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], volume[MAX_NAME_LEN];

        ret = __check_arguments(argc, 0);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_chunkinfo(pool, volume, 0, volume_cmd_data.pithy);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_fsstat(int argc, char **argv)
{
        int ret, retry = 0;
        char *_pool = NULL;

        (void) argv;

        ret = __check_arguments(argc, 0);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _pool = argv[argc - 1];
retry:
        ret = utils_fsstat(_pool);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

#if 0
static int __stor_liststoragearea(int argc, char **argv)
{
        int ret;

        (void) argv;

        ret = __check_arguments(argc, -1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        ret = utils_list_storagearea();
        if (unlikely(ret)) {
                fprintf(stderr,"error: %s\n",strerror(ret));
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}
#endif

static int __stor_cat(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN],
             protocol[MAX_NAME_LEN], volume[MAX_NAME_LEN];

        ret = __check_arguments(argc, 0);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 1];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = utils_cat(pool, volume, volume_cmd_data.offset, volume_cmd_data.length);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_write(int argc, char **argv)
{
        int ret, retry = 0;
        char *_path = NULL, pool[MAX_NAME_LEN], protocol[MAX_NAME_LEN],
             volume[MAX_NAME_LEN],  buf[BIG_BUF_LEN];

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _path = argv[argc - 2];

        ret = path_prep_for_volume(_path, pool, protocol, volume);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        strcpy(buf, argv[argc - 1]);

retry:
        ret = utils_write(pool, buf, volume, volume_cmd_data.offset);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 8, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_authset(int argc, char **argv)
{
        int ret, retry = 0;
        char *_key, *_user, *_passwd;
        char _value[MAX_NAME_LEN];

        ret = __check_arguments(argc, 3);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _key = argv[argc - 3];
        _user = argv[argc - 2];
        _passwd = argv[argc - 1];

        memset(_value, 0x00, MAX_NAME_LEN);
        strcpy(_value, _user);
        strcat(_value, " ");
        strcat(_value, _passwd);
retry:
        ret = lich_auth_set(_key, _value);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_authupdate(int argc, char **argv)
{
        int ret, retry = 0;
        char *_key, *_user, *_passwd;
        char _value[MAX_NAME_LEN];

        ret = __check_arguments(argc, 3);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _key = argv[argc - 3];
        _user = argv[argc - 2];
        _passwd = argv[argc - 1];

        memset(_value, 0x00, MAX_NAME_LEN);
        strcpy(_value, _user);
        strcat(_value, " ");
        strcat(_value, _passwd);
retry:
        ret = lich_auth_update(_key, _value);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_authget(int argc, char **argv)
{
        int ret, retry = 0;
        char *_key;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _key = argv[argc - 1];
retry:
        ret = lich_auth_get(_key);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_authremove(int argc, char **argv)
{
        int ret, retry = 0;
        char *_key;

        ret = __check_arguments(argc, 1);
        if (ret) {
                GOTO(err_ret, ret);
        }

        _key = argv[argc - 1];
retry:
        ret = lich_auth_remove(_key);
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

static int __stor_authlist(int argc, char **argv)
{
        int ret, retry = 0;

        (void) argv;

        ret = __check_arguments(argc, 0);
        if (ret) {
                GOTO(err_ret, ret);
        }

retry:
        ret = lich_auth_list();
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else
                        GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}

#if 0
static int __stor_snap_copy(const char *_snappath, const char *to)
{
        int ret;
        char snappath[MAX_NAME_LEN];

        if (!__is_valid_snappath(_snappath)) {
                ret = EINVAL;
                fprintf(stderr, "snap path was invalid\n");
                GOTO(err_ret, ret);
        }
        __add_lichbd_pre(snappath, _snappath);

        ret = utils_snapshot_copy(snappath, to);
        if (unlikely(ret)) {
                GOTO(err_ret, ret);
        }

        return 0;
err_ret:
        return ret;
}
#endif

static void usage(const struct command *commands, int status)
{
        int i;
        const struct subcommand *s;
        char name[64];
        char text[MAX_NAME_LEN];

        if (status) {
                fprintf(stderr, "Try '%s help' for more information.\n", program_name);
        } else {
                printf("lichbd (version %s)\n", "0.0.2");
                printf("Usage: %s [OPTIONS] <cmd> ...\n", program_name);
                printf("\nAvailable commands:\n");
                for (i = 0; commands[i].name; i++) {
                        for (s = commands[i].sub; s->name; s++) {
                                if (!strcmp(s->name, CMD_BR)) {
                                        printf("\n");
                                        continue;
                                }
                                if (!strcmp(commands[i].name, "snap") &&
                                                ((!strcmp(s->name, "diff")) || (!strcmp(s->name, "cat")))) {
                                        continue;
                                }

                                if (!strcmp(commands[i].name, "path") &&
                                                ((!strcmp(s->name, "cat")) || (!strcmp(s->name, "write")))) {
                                        continue;
                                }

                                if (!strcmp(commands[i].name, system_cmd)) {
                                        snprintf(name, sizeof(name), "%s", s->name);
                                } else {
                                        snprintf(name, sizeof(name), "%s %s",
                                                        commands[i].name, s->name);
                                }


                                sprintf(text, "%s %s", name, s->arg);
                                if (strlen(text) >= 48) {
                                        printf("  %s\n", text);
                                        printf("  %-48s%s\n", "", s->desc);
                                } else {
                                        printf("  %-48s%s\n", text, s->desc);
                                }
                        }
                        // add empty line
                        printf("\n");
                }
                printf("\n");
                printf("supported protocol: iscsi, lichbd, nbd, nfs\n");
                printf("if protocol is iscsi, len(iqn) + len(pool) + len(vol) + 2 must be less than 223\n");
                printf("\tand only accept one layer of pool, character only allow 'a-z', '0-9', '.', '-', ':'\n");
                printf("For more information, run "
                                "'%s <command> <subcommand> help'.\n", program_name);
        }
        exit(status);
}

static struct sd_option pool_options[] = {
        { 0, NULL, false, NULL, NULL},
};

static struct sd_option volume_options[] = {
        {'1', "force", false, "force", NULL},
        {'A', "area", true, "storage area", NULL},
        {'S', "single", true, "do not use multi thread", "do not use mult thread"},
        {'s', "size", true, "size of volume for create and resize", NULL},
        {'c', "concurrent", true, "how many concurrent, limit 100.", NULL},
        {'p', "pipeline", true, "how many pipeline, limit 1000.", NULL},
        {'P', "path", true, "path name for import/export", NULL},
        {'i', "volume", true, "volume name", NULL},
        {'I', "volume-format", true, "volume format: [1|2]", NULL},
        {'C', "type", true, "full info", NULL},
        {'f', "format", true, "output format, such as json", NULL},
        {'F', "lsv", true, "[raw | row2 | row3 | lsv]", NULL},
        {'t', "thin", true, "thin import: [0|1]", NULL},
        {'T', "tier", true, "tier: [0|1]", NULL},
        {'r', "repnum", true, "repnum", NULL},
        { 0, NULL, false, NULL, NULL},
};

static struct sd_option snap_options[] = {
        {'a', "all", false, "show all snap, include deleted", "show all snap, include deleted"},
        {'f', "format", true, "output format, such as json | tree", NULL},
        {'T', "tier", true, "tier: [0|1]", NULL},
        { 0, NULL, false, NULL, NULL},
};

static struct sd_option attr_options[] = {
        {'f', "format", true, "output format: default | json", NULL},
        { 0, NULL, false, NULL, NULL},
};

static struct sd_option path_options[] = {
        {'r', "recursive", false, "recursive output", NULL},
        {'o', "offset", true, "offset", NULL},
        {'l', "lenth", true, "lenth", NULL},
        {'p', "pithy", true, "pithy", NULL},
        {'f', "format", true, "output format, such as json", NULL},
        { 0, NULL, false, NULL, NULL},
};

static struct sd_option map_options[] = {
        {'f', "format", true, "output format: default | json", NULL},
        { 0, NULL, false, NULL, NULL},
};
/*static struct sd_option snap_options[] = {*/
/*{ 0, NULL, false, NULL },*/
/*};*/

static struct subcommand pool_cmd[] = {
        /*没用*/
        {"create", "<pool>  [-e k+r]", "", "create pool",
                NULL, CMD_NEED_ARG,
                __stor_mkpool, pool_options},
        {"rm", "<pool> ", "", "remove pool",
                NULL, CMD_NEED_ARG,
                __stor_rmpool, pool_options},
        {"ls", "[pool] ", "", "list pools",
                NULL, CMD_NEED_NULL,
                __stor_lspools, pool_options},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

const char copy_help[] = "<src> <dist>\n\n \
                          example of <src> <dist>: \n \
                          <pool/src-volume>       <pool/dist-volume>\n \
                          </path/to/local-file>  <pool/dist-volume>\n \
                          <pool/src-volume>       </path/to/local-file\n \
                          ";

static struct subcommand dir_cmd[] = {
        {"create", "<dir> ", "", "create dir",
                NULL, CMD_NEED_ARG,
                __stor_mkdir, pool_options},
        {"rm", "<dir> ", "", "remove dir",
                NULL, CMD_NEED_ARG,
                __stor_rmdir, pool_options},
        {"ls", "[dir] ", "", "list dirs",
                NULL, CMD_NEED_NULL,
                __stor_lsdir, pool_options},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand volume_cmd[] = {
        {"create", "<pool/protocol/volume> --size <M> --repnum num [-T <0|1>]",
                "PTsIArF",
                "create an empty volume, unit(k|K, |m|M, g|G)",
                NULL, CMD_NEED_ARG,
                __stor_create, volume_options},
        {"rm", "<pool/protocol/volume>", "1", "remove volume",
                NULL, CMD_NEED_ARG,
                __stor_rm, volume_options},
        {"ls", "<pool/protocol>", "f", "list volumes",
                NULL, CMD_NEED_ARG,
                __stor_ls, volume_options},
        {"info", "<pool/protocol/volume> --type <allocate|full>", "pfC", "show information about chknum, allocated, etc",
                NULL, CMD_NEED_ARG,
                __stor_info, volume_options},
        {"allocate", "<pool/protocol/volume> [-p] [-c]", "pc",
                "Allocate storage space to the file, [-p] pipeline limit 1000, [-c] concurrent limit 100",
                 NULL, CMD_NEED_ARG,
                 __stor_allocate, volume_options},
        {"resize", "<pool/protocol/volume> --size <M>", "s", "resize the volume, unit(k|K, |m|M, g|G)",
                NULL, CMD_NEED_ARG,
                __stor_resize, volume_options},
        {"mv", "<src> <dest>", "", "move src volume to dest",
                NULL, CMD_NEED_ARG,
                __stor_mv, volume_options},
        {"rename", "<src> <dest>", "", "volume rename. Note: don't supported cross pool",
                NULL, CMD_NEED_ARG,
                __stor_mv, volume_options},
        {"migrate", "<pool/protocol/volume> <pool/protocol/volume>", "1A",
                "migrate volume to specified protection domain.",
                NULL, CMD_NEED_ARG,
                __stor_migrate, volume_options},
        {"copy", "<src> <dest> [-T <0|1>] [-S single]", "TSAF", "copy src volume to dest",
                NULL, CMD_NEED_ARG,
                __stor_copy, volume_options},
        {"duplicate", "<src> <dest>", "A",
                "copy src volume to dest, implemented based on Snapshot",
                NULL, CMD_NEED_ARG,
                __stor_duplicate, volume_options},
        {"import", "<path> <pool/protocol/volume> [-t <1|0>] [-T <0|1>]", "tTIAF",
                "import volume from file. '-' for stdin, -t thin -T tier",
                NULL, CMD_NEED_ARG,
                __stor_import, volume_options},
        {"export", "<pool/protocol/volume> <path>", "", "export volume to file. '-' for stdout",
                NULL, CMD_NEED_ARG,
                __stor_export, volume_options},
        {"lock", "<pool/protocol/volume>", "", "lock an volume",
                NULL, CMD_NEED_ARG,
                __stor_lock, volume_options},
        {"unlock", "<pool/protocol/volume>", "", "unlock an volume",
                NULL, CMD_NEED_ARG,
                __stor_unlock, volume_options},
        {"md5sum", "<pool/protocol/volume>", "S", "calculate MD5",
                NULL, CMD_NEED_ARG,
                __stor_md5sum, volume_options},
        {"connection", "<pool/protocol/volume>", "", "display mapping relation",
                NULL, CMD_NEED_NULL,
                __stor_connection, NULL},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand snap_cmd[] = {
        {"create", "<pool/protocol/volume>@<snap>", "", "create a snapshot",
                NULL, CMD_NEED_ARG,
                __stor_snap_create, snap_options},
        {"rm", "<pool/protocol/volume>@<snap>", "", "deletes a snapshot",
                NULL, CMD_NEED_ARG,
                __stor_snap_rm, snap_options},
        {"ls", "<pool/protocol/volume> [-a]", "fa", "dump list of volume snapshots, -a will show all",
                NULL, CMD_NEED_ARG,
                __stor_snap_ls, snap_options},
        {"rollback", "<pool/protocol/volume>@<snap>", "", "rollback volume to snapshot",
                NULL, CMD_NEED_ARG,
                __stor_snap_rollback, snap_options},
        {"clone", "<pool/protocol/volume>@<snap> <pool/protocol/volume> [-T <0|1>]", "T",
                "clone a snapshot into a COW",
                NULL, CMD_NEED_ARG,
                __stor_snap_clone, snap_options},
        {"flat", "<pool/protocol/volume>", "", "flat a volume ",
                NULL, CMD_NEED_ARG,
                __stor_snap_flat, snap_options},
        {"protect", "<pool/protocol/volume>@<snap>", "", "prevent a snapshot from being deleted",
                NULL, CMD_NEED_ARG,
                __stor_snap_protect, snap_options},
        {"unprotect", "<pool/protocol/volume>@<snap>", "", "allow a snapshot to be deleted",
                NULL, CMD_NEED_ARG,
                __stor_snap_unprotect, snap_options},
#if 1
        // used in make test
        {"cat", "<pool/protocol/volume>@<snap>", "", "read a snapshot ",
                NULL, CMD_NEED_ARG,
                __stor_snap_cat, snap_options},
        {"export", "<pool/protocol/volume>@<snap> [idx|from~to] :<file name>", "", "export snap as file name",
                NULL, CMD_NEED_ARG,
                __stor_snap_export, snap_options},
#endif
        {"diff", "<pool/protocol/volume>@<snap src>~<snap dst> [idx|from~to] :<file name>", "", "diff snap and file name ",
                NULL, CMD_NEED_ARG,
                __stor_snap_diff, snap_options},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand attr_cmd[] = {
        {"set", "<pool|volume> <key> <value>", "", "set attr",
                NULL, CMD_NEED_ARG,
                __stor_attrset, attr_options},
        {"rm", "<pool|volume> <key>", "", "remove attr",
                NULL, CMD_NEED_ARG,
                __stor_attrremove, attr_options},
        {"ls", "<pool|volume>", "", "list all attrs",
                NULL, CMD_NEED_ARG,
                __stor_attrlist, attr_options},
        {"get", "<pool|volume> <key>]", "f", "get attr",
                NULL, CMD_NEED_ARG,
                __stor_attrget, attr_options},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand path_cmd[] = {
        {"find", "<path> [-r recursive]", "r", "show all files in the path",
                NULL, CMD_NEED_ARG,
                __stor_find, path_options},
        {"stat", "<path>", "f", "show the path state",
                NULL, CMD_NEED_ARG,
                __stor_stat, path_options},
        {"cat", "<path> [-o offset] [-l lenth]", "ol", "cat a filename",
                NULL, CMD_NEED_ARG,
                __stor_cat, path_options},
        {"write", "<path> <content> [-o offset]", "o", "write content to path",
                NULL, CMD_NEED_ARG,
                __stor_write, path_options},
        {"fsstat", "<pool>", "", "show the storage system state",
                NULL, CMD_NEED_ARG,
                __stor_fsstat, path_options},
        {"chunkinfo", "<path> [--pithy <0|1>]", "p", "show chunk info, set pithy to 1 for detail.",
                NULL, CMD_NEED_ARG,
                __stor_chunkinfo, path_options},
#if 0
        {"liststoragearea", "", "", "list the storage area",
                NULL, CMD_NEED_ARG,
                __stor_liststoragearea, path_options},
#endif
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand map_cmd[] = {
        {"create", "<initiator> <pool/protocol/volume>", "", "host mapping with volume",
                NULL, CMD_NEED_NULL,
                __stor_createmap, NULL},
        {"rm", "<initiator> <pool/protocol/volume>", "", "host unmapped with volume",
                NULL, CMD_NEED_NULL,
                __stor_rmmap, NULL},
        {"rms", "<initiator> ", "", "rm host map",
                NULL, CMD_NEED_NULL,
                __stor_rmall, NULL},
        {"ls", "<initiator> ", "f", "show host mapping with volume",
                NULL, CMD_NEED_ARG,
                __stor_listmap, map_options},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static struct subcommand auth_cmd[] = {
        {"set", "<initiator> <user> <passwd>", "", "setting initiator user and password",
                NULL, CMD_NEED_ARG,
                __stor_authset, NULL},
        {"rm", "<initiator>", "", "remove initiator",
                NULL, CMD_NEED_ARG,
                __stor_authremove, NULL},
        {"update", "<initiator> <user> <passwd>", "", "update initiator user and password",
                NULL, CMD_NEED_ARG,
                __stor_authupdate, NULL},
        {"ls", "", "", "list",
                NULL, CMD_NEED_ARG,
                __stor_authlist, NULL},
        {"get", "<initiator>", "", "get initiator user and password",
                NULL, CMD_NEED_ARG,
                __stor_authget, NULL},
        {NULL, NULL, NULL, NULL, NULL, CMD_NEED_NULL, NULL, NULL},
};

static int pool_parser(int ch, const char *opt)
{
        int ret;
        uint32_t k = 0, r = 0;
        DINFO("ch : %d, opt: %s\n", ch, opt);

        switch (ch) {
                case 'p':
                        strcpy(pool_cmd_data.protocol_name, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.protocol_name_flag = true;
                        break;
                case 'A':
                        ret = utils_check_storagearea(optarg);
                        if (unlikely(ret))
                                GOTO(err_ret, ret);

                        strcpy(volume_cmd_data.storage_area, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.storage_area_flag = true;
                        break;
                case 'e':
                        ret = sscanf(optarg, "%u+%u", &k, &r);
                        if (ret != 2 || !k || !r) {
                                GOTO(err_ret, ret);
                        }

                        volume_cmd_data.ec.plugin = (!k || !r) ? PLUGIN_NULL : PLGUIN_EC_ISA;
                        volume_cmd_data.ec.tech = (!k || !r) ? TECH_NULL : TECH_ISA_SSE;
                        volume_cmd_data.ec.k = k;
                        volume_cmd_data.ec.m = k + r;

                        cmd_flag_data.arguments += 1;
                default:
                        break;
        }

        return 0;
err_ret:
        return ret;
}

static int volume_parser(int ch, const char *opt)
{
        int ret;

        DINFO("ch : %d, opt: %s\n", ch, opt);

        switch (ch) {
                case '1':
                        volume_cmd_data.force = 1;
                        break;
                case 't':
                        if (!strcmp("1", optarg)) {
                                cmd_flag_data.thin_flag = true;
                        } else {
                                cmd_flag_data.thin_flag = false;
                        }
                        cmd_flag_data.arguments += 1;
                        break;
                case 'T':
                        if (!is_zero_one_char(optarg)) {
                                ret = EINVAL;
                                GOTO(err_ret, ret);
                        }

                        cmd_flag_data.priority = atoi(optarg);
                        cmd_flag_data.arguments += 1;
                        break;
                case 'f':
                        if (!strcmp("default", optarg)) {
                                volume_cmd_data.output_format = 0;
                        } else if (!strcmp("json", optarg)) {
                                volume_cmd_data.output_format = 1;
                        } else {
				ret = EINVAL;
				GOTO(err_ret, ret);
                        }
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.output_format_flag = true;
                        break;
                case 'C':
                        if (!strcmp("allocate", optarg)) {
                                volume_cmd_data.full = VOL_INFO_ALLOCATE;
                        } else if (!strcmp("full", optarg)) {
                                volume_cmd_data.full = VOL_INFO_FULL;
                        } else {
                                volume_cmd_data.full = VOL_INFO_NORMAL;
                        }
                        cmd_flag_data.arguments += 1;
                        break;
                case 'F':
                        if (!strcmp("row2", optarg)) {
                                volume_cmd_data.volume_format = VOLUME_FORMAT_ROW2;
                        } else if (!strcmp("row3", optarg)) {
                                volume_cmd_data.volume_format = VOLUME_FORMAT_ROW3;
                        } else if (!strcmp("lsv", optarg)) {
                                volume_cmd_data.volume_format = VOLUME_FORMAT_LSV;
                        } else {
                                volume_cmd_data.volume_format = VOLUME_FORMAT_RAW;
                        }
                        cmd_flag_data.arguments += 1;
                        break;
                case 'i':
                        strcpy(volume_cmd_data.volume_name, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.volume_name_flag = true;
                        break;
                case 'I':
                        cmd_flag_data.arguments += 1;
                        break;
                case 's':
                        ret = utils_get_size(optarg, &volume_cmd_data.size);
                        if (unlikely(ret))
                                goto err_ret;
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.size_flag = true;
                        break;
                case 'S':
                        volume_cmd_data.mult_thread = 0;
                        cmd_flag_data.mult_thread_flag = true;
                        cmd_flag_data.arguments += 1;
                        break;
                case 'p':
                        volume_cmd_data.pipeline = atoi(optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.pipeline_flag = true;
                        break;
                case 'c':
                        volume_cmd_data.concurrent = atoi(optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.pipeline_flag = true;
                        break;
                case 'P':
                        path_decode(optarg, volume_cmd_data.pool_name, volume_cmd_data.protocol_name, \
                                        volume_cmd_data.volume_name, NULL);
                        if (strlen(volume_cmd_data.pool_name)) {
                                cmd_flag_data.arguments += 1;
                                cmd_flag_data.pool_name_flag = true;
                        }
                        if (strlen(volume_cmd_data.protocol_name)) {
                                cmd_flag_data.arguments += 1;
                                cmd_flag_data.protocol_name_flag = true;
                        }
                        if (strlen(volume_cmd_data.volume_name)) {
                                cmd_flag_data.arguments += 1;
                                cmd_flag_data.volume_name_flag = true;
                        }
                        break;
                case 'A':
                        ret = utils_check_storagearea(optarg);
                        if (unlikely(ret))
                                GOTO(err_ret, ret);

                        strcpy(volume_cmd_data.storage_area, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.storage_area_flag = true;
                        break;
                case 'r':
                        volume_cmd_data.repnum = atoi(optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.repnum_flag = true;
                        break;
                default:
                        break;
        }

        return 0;
err_ret:
        return ret;
}

static int snap_parser(int ch, const char *opt)
{
        int ret;
        DINFO("ch : %d, opt: %s\n", ch, opt);
        switch (ch) {
		case 'a':
			cmd_flag_data.arguments += 1;
			cmd_flag_data.all_flag = true;
                        snap_cmd_data.all = 1;
			break;
		case 'f':
                        if (!strcmp("tree", optarg)) {
                                snap_cmd_data.output_format = 2;
                        } else if (!strcmp("json", optarg)) {
                                snap_cmd_data.output_format = 1;
                        } else if (!strcmp("default", optarg)) {
                                snap_cmd_data.output_format = 0;
                        } else {
				ret = EINVAL;
				GOTO(err_ret, ret);
                        }
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.output_format_flag = true;
                        break;
                case 'i':
                        strcpy(snap_cmd_data.volume_name, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.volume_name_flag = true;
                        break;
                case 'p':
                        strcpy(snap_cmd_data.protocol_name, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.protocol_name_flag = true;
                        break;
                case 'A':
                        ret = utils_check_storagearea(optarg);
                        if (unlikely(ret))
                                GOTO(err_ret, ret);

                        strcpy(volume_cmd_data.storage_area, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.storage_area_flag = true;
                        break;
                case 'T':
                        if (!is_zero_one_char(optarg)) {
                                ret = EINVAL;
                                GOTO(err_ret, ret);
                        }

                        cmd_flag_data.priority = atoi(optarg);
                        cmd_flag_data.arguments += 1;
                        break;
                default:
                        break;
        }

        return 0;
err_ret:
        return ret;
}

static int attr_parser(int ch, const char *opt)
{
        int ret;

        DINFO("ch : %d, opt: %s\n", ch, opt);
        switch (ch) {
                case 'f':
                        if (!strcmp("default", optarg)) {
                                cmd_flag_data.output_format = 0;
                        } else if (!strcmp("json", optarg)) {
                                cmd_flag_data.output_format = 1;
                        } else {
                                ret = EINVAL;
                                GOTO(err_ret, ret);
                        }
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.output_format_flag = true;
                        break;
                case 'p':
                        strcpy(volume_cmd_data.protocol_name, optarg);
                        cmd_flag_data.arguments += 1;
                        cmd_flag_data.protocol_name_flag = true;
                        break;
                default:
                        break;
        }

        return 0;
err_ret:
        return ret;
}

static int path_parser(int ch, const char *opt)
{
        int ret;

        DINFO("ch : %d, opt: %s\n", ch, opt);
        switch (ch) {
                case 'r':
                        cmd_flag_data.recursive_flag = true;
                        break;
                case 'o':
                        cmd_flag_data.offset_flag = true;
                        cmd_flag_data.arguments += 1;
                        volume_cmd_data.offset = atoll(optarg);
                        break;
                case 'l':
                        cmd_flag_data.length_flag = true;
                        cmd_flag_data.arguments += 1;
                        volume_cmd_data.length = atoi(optarg);
                        break;
                case 'p':
                        cmd_flag_data.pithy_flag = true;
                        cmd_flag_data.arguments += 1;
                        volume_cmd_data.pithy = atoi(optarg);
                        break;
                case 'f':
                        if (!strcmp("default", optarg)) {
                                cmd_flag_data.output_format = 0;
                        } else if (!strcmp("json", optarg)) {
                                cmd_flag_data.output_format = 1;
                        } else {
				ret = EINVAL;
				GOTO(err_ret, ret);
                        }
                        cmd_flag_data.output_format_flag = true;
                        cmd_flag_data.arguments += 1;
                        break;
                default:
                        break;
        }

        return 0;
err_ret:
        return ret;
}

static int map_parser(int ch, const char *opt)
{
        int ret;

        DINFO("ch : %d, opt: %s\n", ch, opt);
        switch (ch) {
        case 'f':
                if (!strcmp("default", optarg)) {
                        cmd_flag_data.output_format = 0;
                } else if (!strcmp("json", optarg)) {
                        cmd_flag_data.output_format = 1;
                } else {
                        ret = EINVAL;
                        GOTO(err_ret, ret);
                }
                cmd_flag_data.arguments += 1;
                cmd_flag_data.output_format_flag = true;
                break;
        default:
                break;
        }

        return 0;
err_ret:
        return ret;
}

// 分析程序结构的切入点: 先定义规则，然后解析，执行
struct command pool_command = {
        "pool",
        pool_cmd,
        pool_parser
};

struct command dir_command = {
        "dir",
        dir_cmd,
        pool_parser
};

struct command volume_command = {
        "vol",
        volume_cmd,
        volume_parser
};

struct command snap_command = {
        "snap",
        snap_cmd,
        snap_parser
};

struct command attr_command = {
        "attr",
        attr_cmd,
        attr_parser
};

struct command map_command = {
        "map",
        map_cmd,
        map_parser,
};

struct command auth_command = {
        "auth",
        auth_cmd,
        NULL,
};

struct command path_command = {
        "path",
        path_cmd,
        path_parser
};

static const struct sd_option *find_opt(int ch)
{
        const struct sd_option *opt;

        /* search for common options */
        sd_for_each_option(opt, lichbd_options) {
                if (opt->ch == ch)
                        return opt;
        }

        /* search for self options */
        if (command_options) {
                sd_for_each_option(opt, command_options) {
                        if (opt->ch == ch)
                                return opt;
                }
        }

        fprintf(stderr, "Internal error\n");
        exit(1);
}

static void init_commands(const struct command **commands)
{
        int ret;
        static struct command *cmds;
        struct command command_list[] = {
                pool_command,
                dir_command,
                volume_command,
                snap_command,
                attr_command,
                map_command,
                auth_command,
                path_command,
                {NULL, NULL, NULL},
        };

        if (!cmds) {
                ret = ymalloc((void **)&cmds, sizeof(command_list));
                if (unlikely(ret)) {
                        YASSERT(0);
                }
                memcpy(cmds, command_list, sizeof(command_list));
        }

        *commands = cmds;
        return;
}

static const struct subcommand *find_subcmd(const char *cmd, const char *subcmd)
{
        int i, j;
        const struct command *commands;
        const struct subcommand *sub;

        init_commands(&commands);

        for (i = 0; commands[i].name; i++) {
                if (!strcmp(commands[i].name, cmd)) {
                        sub = commands[i].sub;
                        for (j = 0; sub[j].name; j++) {
                                if (!strcmp(sub[j].name, subcmd))
                                        return &sub[j];
                        }
                }
        }

        return NULL;
}

static unsigned long setup_commands(const struct command *commands,
                const char *cmd, const char *subcmd)
{
        int i;
        bool found = false;
        const struct subcommand *s;
        unsigned long flags = 0;

        for (i = 0; commands[i].name; i++) {
                if (!strcmp(commands[i].name, cmd)) {
                        found = true;
                        if (commands[i].parser)
                                command_parser = commands[i].parser;
                        break;
                }
        }
        if (!found) {
                for (i = 0; commands[i].name; i++) {
                        if (!strcmp(commands[i].name, system_cmd)) {
                                break;
                        }
                }

                for (s = commands[i].sub; s->name; s++) {
                        if (!strcmp(s->name, cmd)) {
                                found = true;
                                found_system_cmd = true;
                                if (commands[i].parser)
                                        command_parser = commands[i].parser;
                                break;
                        }
                }
        }

        if (!found) {
                if (cmd && strcmp(cmd, "help") && strcmp(cmd, "--help") &&
                                strcmp(cmd, "-h")) {
                        fprintf(stderr, "Invalid command '%s'\n", cmd);
                        usage(commands, EINVAL);
                }
                usage(commands, 0);
        }

        for (s = commands[i].sub; (!found_system_cmd?subcmd:cmd) && s->name; s++) {
                if (!strcmp(s->name, (!found_system_cmd?subcmd:cmd))) {
                        command_fn = s->fn;
                        command_opts = s->opts;
                        command_arg = s->arg;
                        command_desc = s->desc;
                        command_options = s->options;
                        flags = s->flags;
                        break;
                }
        }

        if (!command_fn) {
                if (found_system_cmd) {
                        usage(commands, 0);
                }

                if (subcmd && strcmp(subcmd, "help") &&
                                strcmp(subcmd, "--help") && strcmp(subcmd, "-h"))
                        fprintf(stderr, "Invalid command '%s %s'\n", cmd, subcmd);

                fprintf(stderr, "Available %s commands:\n", cmd);
                for (s = commands[i].sub; s->name; s++) {
                        if (!strcmp(s->name, CMD_BR)) {
                                printf("\n");
                                continue;
                        }
                        if (!strcmp(s->name, "diff")) {
                                continue;
                        }

                        fprintf(stderr, "  %s %s\n", cmd, s->name);
                }
                exit(EINVAL);
        }

        return flags;
}

void subcommand_usage(char *cmd, char *subcmd, int status)
{
        int i, n, len = strlen(command_opts);
        const struct sd_option *sd_opt;
        const struct subcommand *sub, *subsub;
        char name[64];
        char text[MAX_NAME_LEN];

        printf("Usage: %s %s %s", program_name, found_system_cmd?"":cmd, subcmd);

        if (0 <= subcmd_depth) {
                for (i = 0; i < subcmd_depth + 1; i++)
                        printf(" %s", subcmd_stack[i]->name);

                subsub = subcmd_stack[i - 1]->sub;
        } else {
                sub = find_subcmd(cmd, subcmd);
                subsub = sub->sub;
        }

        if (subsub) {
                n = 0;
                while (subsub[n].name)
                        n++;
                if (n == 1)
                        printf(" %s", subsub[0].name);
                else if (n > 1) {
                        printf(" {%s", subsub[0].name);
                        for (i = 1; i < n; i++)
                                printf("|%s", subsub[i].name);
                        printf("}");
                }
        }

        /**
        for (i = 0; i < len; i++) {
                sd_opt = find_opt(command_opts[i]);
                if (sd_opt->has_arg) {
                        if (sd_opt->ch != 's')
                                printf(" [-%c %s]", sd_opt->ch, sd_opt->name);
                }
                else
                        printf(" [-%c]", sd_opt->ch);
        }
        **/

        if (command_arg)
                printf(" %s", command_arg);

        printf("\n");
        if (subsub) {
                printf("Available subcommands:\n");
                for (i = 0; subsub[i].name; i++) {
                        sprintf(text, "%s %s", subsub[i].name, subsub[i].arg);
                        if (strlen(text) >= 45) {
                                printf("  %s\n", text);
                                printf("  %-45s%s\n", "", subsub[i].desc);
                        } else {
                                printf("  %-45s%s\n", text, subsub[i].desc);
                        }
                }

        }

        for (i = 0; i < len; i++) {
                if (i == 0)
                        printf("Options:\n");
                sd_opt = find_opt(command_opts[i]);
                if (sd_opt->ch != '1') {
                        snprintf(name, sizeof(name), "-%c, --%s",
                                sd_opt->ch, sd_opt->name);
                        printf("  %-24s%s\n", name, sd_opt->desc);
                }
        }

        exit(status);
}

static const struct sd_option *build_sd_options(const char *opts)
{
        static struct sd_option sd_opts[256], *p;
        int i, len = strlen(opts);

        p = sd_opts;
        for (i = 0; i < len; i++)
                *p++ = *find_opt(opts[i]);
        memset(p, 0, sizeof(struct sd_option));

        return sd_opts;
}

static int format_args(int argc, char *argv[], char *new[])
{
        int i, c = 0, c1 = 0;
        char *new1[256];

        for (i = 0; i < argc; i++) {
                if ('-' == argv[i][0]) {
                        new1[c1] = (char *)malloc(256);
                        strcpy(new1[c1++], argv[i++]);
                        if (NULL != argv[i]) {
                                new1[c1] = (char *)malloc(256);
                                strcpy(new1[c1++], argv[i]);
                        }
                } else {
                        new[c] = (char*)malloc(256);
                        strcpy(new[c++], argv[i]);
                }
        }

        for (i = 0; i < c1; i++) {
                new[i + c] = (char*)malloc(256);
                strcpy(new[i + c], new1[i]);
                free(new1[i]);
        }

        return 0;
}

static void free_args(int argc, char *new[])
{
        int i;

        for (i = 0; i < argc; i++) {
                free(new[i]);
        }
}

int main(int argc, char *argv[])
{
        int ch, longindex, ret, retry = 0;
        unsigned long flags;
        struct option *long_options;
        const struct command *commands;
        const char *short_options;
        const struct sd_option *sd_opts;
        bool verbose;
        char argument1[256], argument2[256];
        char *argvnew[256];

        (void) verbose;
        
        dbg_info(0);
        volume_cmd_data.mult_thread = 1;
        volume_cmd_data.pipeline = ALLOCATE_PIPE_DEFAULT;
        volume_cmd_data.concurrent = ALLOCATE_THREAD_DEFAULT;
        volume_cmd_data.output_format = 0;
        volume_cmd_data.volume_format = VOLUME_FORMAT_RAW;
        volume_cmd_data.offset = 0;
        volume_cmd_data.length = 0;
        volume_cmd_data.full = VOL_INFO_NORMAL;
        volume_cmd_data.force = 0;
        strcpy(volume_cmd_data.volume_name, "");
        strcpy(volume_cmd_data.protocol_name, "");
        strcpy(volume_cmd_data.path, "");
        volume_cmd_data.size = 0;
        volume_cmd_data.repnum = 0;
        strcpy(volume_cmd_data.storage_area, STORAGE_AREA_VALUE_NULL);
        strcpy(snap_cmd_data.storage_area, STORAGE_AREA_VALUE_NULL);
        volume_cmd_data.ec.plugin = PLUGIN_NULL;

        init_commands(&commands);

        if (argc == 1) {
                usage(commands, 0);
                exit(EINVAL);
        }

        format_args(argc, argv, argvnew);

        if (argc > 2) {
                strcpy(argument1, argvnew[1]);
                strcpy(argument2, argvnew[2]);
                flags = setup_commands(commands, argvnew[1], argvnew[2]);
        } else {
                strcpy(argument1, argvnew[1]);
                flags = setup_commands(commands, argvnew[1], NULL);
        }

        free_args(argc, argvnew);

        optind = 3?!found_system_cmd:2;

        sd_opts = build_sd_options(command_opts);
        long_options = build_long_options(sd_opts);
        short_options = build_short_options(sd_opts);

        //TODO
        ret = env_init_simple("lichbd");
        if (unlikely(ret))
                GOTO(err_ret, ret);

retry:
        ret = network_connect_master();
        if (unlikely(ret)) {
                ret = _errno(ret);
                if (ret == EAGAIN || ret == ENOSPC) {
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.rpc_timeout * 4, (1000 * 1000));
                } else if (ret == EHOSTDOWN) {
                        DWARN("master not online\n");
                        USLEEP_RETRY(err_ret, ret, retry, retry, gloconf.master_timeout + 2, (1000 * 1000));
                        GOTO(err_ret, ret);
                }
                        GOTO(err_ret, ret);
        }

        ret = stor_init(NULL, -1);
        if (unlikely(ret))
                GOTO(err_ret, ret);

        while ((ch = getopt_long(argc, argv, short_options, long_options,
                                        &longindex)) >= 0) {
                switch (ch) {
                        case 'v':
                                verbose = true;
                                break;
                        case 'h':
                                subcommand_usage(argv[1], argv[2], 0);
                                break;
                        case '?':
                                usage(commands, EINVAL);
                                break;
                        default:
                                if (command_parser) {
                                        ret = command_parser(ch, optarg);
                                        if (unlikely(ret))
                                                GOTO(err_ret, ret);
                                }
                                else
                                        usage(commands, EINVAL);
                                break;
                }
        }

#if 0
        if (verbose) {
                dbg_goto(1);
        } else {
                dbg_goto(0);
        }
#endif

        if (flags & CMD_NEED_ARG && argc == optind) {
                if (found_system_cmd) {
                        subcommand_usage(system_cmd, argument1, EINVAL);
                } else {
                        subcommand_usage(argument1, argument2, EINVAL);
                }
        }

        int i;
        for (i = 0; i< argc; i++) {
                if (!strcmp(argv[i], "help")) {
                        if (found_system_cmd) {
                                subcommand_usage(system_cmd, argument1, 0);
                        } else {
                                subcommand_usage(argument1, argument2, 0);
                        }
                }
        }

        ret = command_fn(argc, argv);
        if (unlikely(ret)) {
                if (ret == EINVAL) {
                        if (found_system_cmd) {
                                subcommand_usage(system_cmd, argument1, EINVAL);
                        } else {
                                subcommand_usage(argument1, argument2, EINVAL);
                        }
                } else {
                        GOTO(err_ret, ret);
                }
        }

        return 0;
err_ret:
        EXIT(_errno(ret));
}
